#include "WebPageQuery.h"
#include "json.hpp"
#include <iostream>
#include <fstream>
#include <sstream>
#include <algorithm>
#include <iterator>
#include <cmath>
using json = nlohmann::json;

//计算点积
static double dotProduct(const vector<double>& a, const vector<double>& b){
    double result = 0.0;
    for (size_t i = 0; i < a.size(); ++i){
        result += a[i] * b[i];
    }
    return result;
}

//计算范数
static double norm(const vector<double>& v){
    double sum = 0.0;
    for (double value : v){
        sum += value * value;
    }
    return sqrt(sum);
}

//计算两个向量的余弦值
static double cosineSimilarity(const vector<double>& a, const vector<double>& b){
    if (a.size() != b.size()){
        throw invalid_argument("Vectors must be of the same dimension.");
    }
    double dot = dotProduct(a, b);
    double normA = norm(a);
    double normB = norm(b);
    return dot / (normA * normB);
}


WebPageQuery::WebPageQuery(Configuration *config, SplitTool &splitTool)
: _config(config), _splitTool(splitTool){

}

WebPageQuery::~WebPageQuery(){

}


string WebPageQuery::doQuery(const string &sentence){
    //读取磁盘文件
    loadLibary();
    /* cout << _pageLib.size() << endl; */
    cout << "读取磁盘文件成功\n";
    
    //对用户输入的语句进行分词和去停用词操作
    vector<string> words = _splitTool.cut(sentence);

    //去停用词
    set<string> &stopWords = _config->getStopWordList();
    words.erase(remove_if(words.begin(), words.end(), [stopWords](string word){
        return stopWords.count(word) != 0;
    }), words.end());

    //如果去停用词没用剩余的单词，直接返回空字符串
    if(words.size() == 0){
        json emptyJson;
        return emptyJson.dump();
    }
    cout << "去停用词成功\n";

    //计算基准向量
    vector<double> baseWeights = getQueryWordsWeightVector(words);
    /* cout << "baseWeights.size() = " << baseWeights.size() << endl; */
    cout << "计算基准向量成功\n";

    //计算交集中文档各单词的权重
    vector<pair<int, vector<double>>> wordWeights;
    bool isEmpty = executeQuery(words, wordWeights);
    cout << "计算交集中文档各单词的权重成功\n";
    
    //如果交集大小等于0，直接返回空字符串
    if(isEmpty){
        json emptyJson;
        return emptyJson.dump();
    }

    //计算每个文档与base向量的余弦值
    vector<pair<int, double>> docWeights;
    for(auto &i: wordWeights){
        double cosine = cosineSimilarity(baseWeights, i.second);
        docWeights.push_back({i.first, cosine});
    }
    sort(docWeights.begin(), docWeights.end(), [](const pair<int, double> &a, const pair<int, double> &b){
         return a.second > b.second;
    });
    cout << "计算每个文档与base向量的余弦值成功\n";
    
    vector<int> docidVec; 
    for(auto &i: docWeights){
        docidVec.push_back(i.first);
    }
    return createJson(docidVec);
}

//加载网页偏移库和倒排索引库
void WebPageQuery::loadLibary(){
    //读网页偏移库
    ifstream ifs1(_config->parseConfig("NEW_OFFSET"));
    string key, pair1, pair2;
    while(ifs1 >> key && ifs1 >> pair1 && ifs1 >> pair2){
        _offsetLib.insert({stoi(key), {stoi(pair1), stoi(pair2)}});
    }
    cout << "读取网页偏移库成功\n";
    
    //读网页倒排索引库
    ifstream ifs2(_config->parseConfig("INVERT_INDEX"));
    string line;
    while(getline(ifs2, line)){
        istringstream iss(line);
        string key, pair1, pair2;
        iss >> key;
        while(iss >> pair1 && iss >> pair2){
            _invertIndexTable[key].insert({stoi(pair1), stoi(pair2)});
        }
    }
    cout << "读取倒排索引库成功\n";
}

vector<double> WebPageQuery::getQueryWordsWeightVector(vector<string> &queryWords){
    //分词后的关键词词组queryWords
    //对其使用TF-IDF算法
    map<string, int> temp;
    for(auto &i: queryWords){
        temp[i]++;
    }
    vector<double> res;
    double IDF = log2(1/2);
    double total = 0;
    for(auto &i: queryWords){
        double w = temp[i] * IDF;
        res.push_back(temp[i] * IDF);
        total += pow(w, 2);
    }
    //计算所有权重平方和的平方根
    total = sqrt(total);
    for(auto &i: res){
        i = i / total;
    }
    return res;
}

bool WebPageQuery::executeQuery(const vector<string> &queryWords, vector<pair<int, vector<double>>> &resultVec){
    //计算所有关键词所在文章id集合的交集
    set<int> inter;
    bool isFirst = true;
    for(auto &word: queryWords){
        set<int> temp;
        for(auto &i: _invertIndexTable[word]){
            temp.insert(i.first);
        }
        if(isFirst){
            inter = temp;
            isFirst = false;
        }
        else{
            set<int> newSet;
            set_intersection(inter.begin(), inter.end(), temp.begin(), temp.end(), inserter(newSet, newSet.begin()));
            inter = newSet;
        }
    }
    /* cout << "交集的大小：" << inter.size() << endl; */
    if(inter.size() == 0){
        return true; //交集不存在返回true
    }
    //计算关键词在所有交集中文档的权重
    map<int, vector<double>> result;
    for(auto &word: queryWords){
        for(auto &i: _invertIndexTable[word]){
            if(inter.count(i.first)){
                result[i.first].push_back(i.second);
            }
        }
    }
    //将map转换为vector
    vector<pair<int, vector<double>>> vec(result.begin(), result.end());
    resultVec = vec;
    return false; //交集存在返回false
}

//生成查询结果并封装为json格式再作为string返回
string WebPageQuery::createJson(vector<int> & docidVec){
    //返回查询结果
    vector<string> result;
    //返回结果时再将网页内容从磁盘组读出，以时间换取空间
    ifstream ifs(_config->parseConfig("NEW_WEBPAGE"));
    for(auto &i: docidVec){
        int pos = _offsetLib[i].first;
        int len = _offsetLib[i].second;
        vector<char> buffer(len);
        ifs.seekg(pos);
        ifs.read(buffer.data(), len);
        string buf(buffer.begin(), buffer.end()); //读出的字符串形式的网页内容
        
        //将字符串封装为json格式
        json jsonStr;

        size_t startPos = 0;
        size_t endPos = 0;
        startPos = buf.find("<title>") + string("<title>").size();
        endPos = buf.find("</title>");
        jsonStr["title"] = buf.substr(startPos, endPos - startPos);
        
        startPos = buf.find("<url>") + string("<url>").size();
        endPos = buf.find("</url>");
        jsonStr["url"] = buf.substr(startPos, endPos- startPos);
        
        startPos = buf.find("<tops>") + string("<tops>").size();
        endPos = buf.find("</tops>");
        jsonStr["tops"] = buf.substr(startPos, endPos - startPos);

        startPos = buf.find("<content>") + string("<content>").size();
        endPos = buf.find("</content>");
        jsonStr["content"] = buf.substr(startPos, endPos - startPos);        
        //再把封装好的json转换为字符串存到字符串数组
        result.push_back(jsonStr.dump());

    }
    //将字符串数组也封装为json格式
    json jsonVec;
    jsonVec["data"] = result;
    //将字符串化的json对象发送给客户端
    return jsonVec.dump();
    
}

